{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from selenium import webdriver \n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-2-89ffedf7a14e>:7: DeprecationWarning: use options instead of chrome_options\n",
      "  driver = webdriver.Chrome( chrome_options = opts)\n"
     ]
    }
   ],
   "source": [
    "opts = webdriver.ChromeOptions()\n",
    "opts.add_argument('--no-sandbox')#解决DevToolsActivePort文件不存在的报错\n",
    "opts.add_argument('window-size=1920x3000') #指定浏览器分辨率\n",
    "opts.add_argument('--disable-gpu') #谷歌文档提到需要加上一这个属性来规避bug\n",
    "opts.add_argument('--hide-scrollbars') #隐藏滚动条, 应对些特殊页面\n",
    "#opts.add_argument('blink-settings=imagesEnabled=false') #不加载图片, 提升速度\n",
    "driver = webdriver.Chrome( chrome_options = opts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "from urllib.parse import urljoin\n",
    "from uuid import uuid4\n",
    "import requests\n",
    "import base64\n",
    "from PIL  import Image"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 调用api"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def base64_api(uname, pwd, img, typeid):\n",
    "    with open(img, 'rb') as f:\n",
    "        base64_data = base64.b64encode(f.read())\n",
    "        b64 = base64_data.decode()\n",
    "    data = {\"username\": uname, \"password\": pwd, \"typeid\": typeid, \"image\": b64}\n",
    "    result = json.loads(requests.post(\"http://api.ttshitu.com/predict\", json=data).text)\n",
    "    if result['success']:\n",
    "        return result[\"data\"][\"result\"]\n",
    "    else:\n",
    "        return result[\"message\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cut_picture(picture_path):\n",
    "    srcPath = picture_path+\".jpg\"\n",
    "    dstPath = picture_path+\".png\"\n",
    "\n",
    "    # 读取图片\n",
    "    img_1 = Image.open(srcPath)\n",
    "    # 设置裁剪的位置\n",
    "    crop_box = (486,354,750,415)\n",
    "    # 裁剪图片\n",
    "    img_2 = img_1.crop(crop_box)\n",
    "    img_2.save(dstPath)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 验证码填入过程\n",
    "![code](./pic/ver.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "已经进入验证码阶段\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\86188\\anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\webdriver.py:1030: UserWarning: name used for saved screenshot does not match file type. It should end with a `.png` extension\n",
      "  warnings.warn(\"name used for saved screenshot does not match file \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "识别结果为： sgwc\n",
      "《电子科技大学学报》征订启事----没有下载权限\n",
      "基于图卷积神经网络的位置语义推断----没有下载权限\n",
      "基于数据挖掘的电网用户行为分析----没有下载权限\n",
      "信息化时代反恐情报工作的创新、挑战与启示:基于对美国的分析----没有下载权限\n",
      "无人车大数据与云控制技术综述                                                                     撤回----没有下载权限\n",
      "基于关联规则结合贝叶斯网络的症状间关系及其对证候诊断贡献度的研究：以新型冠状病毒肺炎湿热蕴肺证为例----没有下载权限\n",
      "基于数据挖掘探讨谢春光治疗糖尿病肾病组方规律----没有下载权限\n",
      "基于中医传承辅助平台探讨中医药治疗支气管扩张的证治规律----没有下载权限\n"
     ]
    }
   ],
   "source": [
    "data = pd.read_excel(\"CNKI_数据挖掘.xlsx\")\n",
    "base = \"https://kns.cnki.net/\"\n",
    "count = 450\n",
    "end_count = 500\n",
    "title_list = data[\"标题\"].to_list()[count:end_count]\n",
    "download_url_list = data[\"链接\"].to_list()[count:end_count]\n",
    "for title,download_url in zip(title_list,download_url_list):\n",
    "    try:\n",
    "        driver.get(download_url)\n",
    "        time.sleep(0.5)\n",
    "        js = 'document.getElementById(\"pdfDown\").setAttribute(\"target\",\"\");'\n",
    "        driver.execute_script(js)\n",
    "        time.sleep(0.5)\n",
    "        driver.find_element_by_xpath('//li[@class=\"btn-dlpdf\"]').click()\n",
    "        time.sleep(0.5)\n",
    "        source = driver.page_source\n",
    "        if \"安全验证\" in source: # 如果安全验证在html内，那么就需要调用api\n",
    "            print(\"已经进入验证码阶段\")\n",
    "            time.sleep(2)\n",
    "            url_path = re.findall('<dt><img id=\"vImg\" src=\"(.*?)\" alt=\"验证码\" title=\"点击切换验证码\"></dt>',source)[0] # 取出图片url\n",
    "            picture_url = urljoin(base,url_path)\n",
    "            content = requests.get(picture_url).content\n",
    "            picture_name = f\"./code/{uuid4()}\"\n",
    "            driver.get_screenshot_as_file(picture_name+\".jpg\")\n",
    "            cut_picture(picture_name)\n",
    "            result = base64_api(uname='#####省去', pwd='#####省去', img=picture_name+\".png\", typeid=3)\n",
    "            print(\"识别结果为：\",result) # 识别出的图片结果\n",
    "            driver.find_element_by_id('vcode').send_keys(result) # 将图片结果填入\n",
    "            time.sleep(5)\n",
    "            driver.find_element_by_xpath('/html/body/div/form/dl/dd/button').click() # 点击提交进行下载\n",
    "    except:\n",
    "        print(f'{title}----没有下载权限')\n",
    "        continue\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 结果\n",
    "![result](./pic/result.png)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
